package urduTermRecognizer;


import java.io.BufferedReader;
import java.io.IOException;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;

public class urduTermRecognizer
{
     ArrayList<String> myWords = new ArrayList<String>();
     ArrayList<String> myTags = new ArrayList<String>();
     
     HashMap<String, Term> Terms = new HashMap<String, Term>();
     
     String CORPUS_INPUT="F:\\Uinput\\"; 
     String CORPUS_OUTPUT="F:\\Uoutput\\";
     
     
     ArrayList<String> mymultiNNCandidate = new ArrayList<String>();
     ArrayList<String> mymultiNNTag = new ArrayList<String>();
     
     ArrayList<String> myJJNNCandidate = new ArrayList<String>();
     ArrayList<String> myJJNNTag = new ArrayList<String>();
        
    ArrayList<String> myNNCCNNCandidate = new ArrayList<String>();
    ArrayList<String> myNNCCNNTag = new ArrayList<String>();
    
    ArrayList<String> myNNCMNNCandidate = new ArrayList<String>();
    ArrayList<String> myNNCMNNTag = new ArrayList<String>();
/*****************************************************************/
public void convert_to_Two_Column_text()
{           
        File dir = new File("F:\\Uinput\\");
        String[] str = dir.list();          

        File inputFile;

        for(String inputFileName : str)
        {
            inputFile = new File(CORPUS_INPUT + inputFileName);
            //System.out.println("\nInput  :  "+inputFile);
           
            String outputFile = CORPUS_OUTPUT+inputFile.getName();
            //System.out.println("Output  :  "+outputFile);

            StringBuffer inputContents = new StringBuffer();
            StringBuffer outputContents = new StringBuffer();

            inputContents =fileReader(inputFile);
            String inputText = inputContents.toString();            
            
            StringBuffer word = new StringBuffer("");
            StringBuffer Tag = new StringBuffer("");
            
            for(int i=0;i< inputText.length();i++)
            {
                while(inputText.charAt(i)!='<')
                {    
                   char c = inputText.charAt(i); 
                    word.append(c);     
                    i++;
                }
                if(inputText.charAt(i)=='<')
                {
                    myWords.add(word.toString());
                    word = new StringBuffer("");
                    i++;
                }
                while(inputText.charAt(i)!='>')
                {                      
                   char c = inputText.charAt(i); 
                    Tag.append(c);     
                    i++;
                }
                 if(inputText.charAt(i)=='>')
                {
                    myTags.add(Tag.toString());
                    Tag = new StringBuffer("");
                    i++;
                }
            }             
            for(int i=0;i< myWords.size();i++)
            {
                 outputContents.append(myWords.get(i)+"\t"+myTags.get(i)+System.getProperty("line.separator"));          
            }
              fileWriter(outputContents,  outputFile);   
        }

    }
/*****************************************************************/
public void extractmultiNNCandidate()
{
    StringBuffer outCand = new StringBuffer();
    String multiNNFile = CORPUS_OUTPUT+"NN+.txt";
      
     for(int i=0;i< myTags.size()-1;i++)
     { 
            if((myTags.get(i).equals("NN") ))
            {                 
                int iloc=i;
                int ilength=0;
                StringBuffer multiNN = new StringBuffer();
                StringBuffer multiNNTag = new StringBuffer();
                while(myTags.get(iloc).equals("NN"))
                {
                    multiNN.append(myWords.get(iloc).trim());
                    multiNN.append(" ");
                    multiNNTag.append(myTags.get(iloc).trim());
                    iloc++;
                    ilength++;
                    if(ilength>0)
                    {
                        mymultiNNTag.add(multiNNTag.toString().trim());
                        mymultiNNCandidate.add(multiNN.toString().trim());
                    }
                }
               }
       }
     
       //for(int i=0;i< mymultiNNTag.size();i++)
     //System.out.println("Multi Candidate : "+mymultiNNCandidate.get(i) +"  Candidate Tag : "+mymultiNNTag.get(i));
      
       for(int i=0;i< mymultiNNCandidate.size();i++)
       {
            outCand.append(mymultiNNCandidate.get(i)+"\t"+mymultiNNTag.get(i)+System.getProperty("line.separator"));          
       }
       fileWriter(outCand,  multiNNFile);
       System.out.println("Total words "+myWords.size()+" Total NN+ : "+mymultiNNTag.size());

}
/*****************************************************************/
public void extractJJNNCandidate()
{
    StringBuffer outCand = new StringBuffer();

    String JJNNFile = CORPUS_OUTPUT+"JJNN+.txt";
  
     for(int i=0,j=1;i< myTags.size()&&j< myTags.size()-1;i++,j++)
     { 
         if((myTags.get(i).equals("JJ") ))
         {  
             StringBuffer JJNNCand = new StringBuffer();
             StringBuffer JJNNTag = new StringBuffer();

             JJNNCand.append(myWords.get(i).trim());
             JJNNCand.append(" ");
             JJNNTag.append(myTags.get(i).trim());
             
            if((myTags.get(j).equals("NN") ))
            {                 
                int iloc=j;
                int ilength=0;
                
                while(myTags.get(iloc).equals("NN"))
                {
                    JJNNCand.append(myWords.get(iloc).trim());
                    JJNNCand.append(" ");
                    JJNNTag.append(myTags.get(iloc).trim());
                    iloc++;
                    ilength++;
                    if(ilength>0)
                    {
                        myJJNNTag.add(JJNNTag.toString().trim());
                        myJJNNCandidate.add(JJNNCand.toString().trim());
                    }
                }
               }
             }
       }
     
     //  for(int i=0;i< myJJNNTag.size();i++)
     //  System.out.println("Candidate : "+myJJNNCandidate.get(i) +"  Candidate Tag : "+myJJNNTag.get(i));
      
       for(int i=0;i< myJJNNCandidate.size();i++)
       {
            outCand.append(myJJNNCandidate.get(i)+"\t"+myJJNNTag.get(i)+System.getProperty("line.separator"));          
       }
       fileWriter(outCand,  JJNNFile);
    
       System.out.println("Total words "+myWords.size()+" Total JJNN+ : "+myJJNNTag.size());


}  
/*****************************************************************/
public void extractNNCCNNCandidate()
{
    StringBuffer outCand = new StringBuffer();
    String NNCCNNFile = CORPUS_OUTPUT+"NNCCNN.txt";
  
     for(int i=0,j=1,k=2;i< myTags.size()&&j< myTags.size()-1&&k< myTags.size()-2;i++,j++,k++)
     { 
         if((myTags.get(i).equals("NN") ))
         {  
             StringBuffer NNCCNNCand = new StringBuffer();
             StringBuffer NNCCNNTag = new StringBuffer();

             NNCCNNCand.append(myWords.get(i).trim());
             NNCCNNCand.append(" ");
             NNCCNNTag.append(myTags.get(i).trim());
             
            if((myTags.get(j).equals("CC") ))
            {                 
              NNCCNNCand.append(myWords.get(j).trim());
              NNCCNNCand.append(" ");
              NNCCNNTag.append(myTags.get(j).trim());
               
                if((myTags.get(k).equals("NN") ))
                {                 
                    NNCCNNCand.append(myWords.get(k).trim());
                    NNCCNNCand.append(" ");
                    NNCCNNTag.append(myTags.get(k).trim());
                 
                        myNNCCNNTag.add(NNCCNNTag.toString().trim());
                        myNNCCNNCandidate.add(NNCCNNCand.toString().trim());
                   
               }
             }
       }
     }
       //for(int i=0;i< myNNCCNNCandidate.size();i++)
       //System.out.println("Candidate : "+myNNCCNNCandidate.get(i) +"  Candidate Tag : "+myNNCCNNTag.get(i));
      
       for(int l=0;l< myNNCCNNCandidate.size();l++)
       {
            outCand.append(myNNCCNNCandidate.get(l)+"\t"+myNNCCNNTag.get(l)+System.getProperty("line.separator"));          
       }
       fileWriter(outCand,  NNCCNNFile);
    
       System.out.println("Total words "+myWords.size()+" Total NNCCNN : "+myNNCCNNTag.size());


}
/*****************************************************************/
public void extractNNCMNNCandidate()
{
    StringBuffer outCand = new StringBuffer();
    String NNCMNNFile = CORPUS_OUTPUT+"NNCMNN.txt";
  
     for(int i=0,j=1,k=2;i< myTags.size()&&j< myTags.size()-1&&k< myTags.size()-2;i++,j++,k++)
     { 
         if((myTags.get(i).equals("NN") ))
         {  
             StringBuffer NNCMNNCand = new StringBuffer();
             StringBuffer NNCMNTag = new StringBuffer();

             NNCMNNCand.append(myWords.get(i).trim());
             NNCMNNCand.append(" ");
             NNCMNTag.append(myTags.get(i).trim());
             
            if((myTags.get(j).equals("CM") ))
            {                 
              NNCMNNCand.append(myWords.get(j).trim());
              NNCMNNCand.append(" ");
              NNCMNTag.append(myTags.get(j).trim());
               
                if((myTags.get(k).equals("NN") ))
                {                 
                    NNCMNNCand.append(myWords.get(k).trim());
                    NNCMNNCand.append(" ");
                    NNCMNTag.append(myTags.get(k).trim());
                 
                        myNNCMNNTag.add(NNCMNTag.toString().trim());
                        myNNCMNNCandidate.add(NNCMNNCand.toString().trim());
                   
               }
             }
       }
     }
     //  for(int i=0;i< myJJNNTag.size();i++)
     //  System.out.println("Candidate : "+myJJNNCandidate.get(i) +"  Candidate Tag : "+myJJNNTag.get(i));
      
       for(int l=0;l< myNNCMNNCandidate.size();l++)
       {
            outCand.append(myNNCMNNCandidate.get(l)+"\t"+myNNCMNNTag.get(l)+System.getProperty("line.separator"));          
       }
       fileWriter(outCand,  NNCMNNFile);
    
       System.out.println("Total words "+myWords.size()+" Total NNCMNN : "+myNNCMNNTag.size());


}
/*****************************************************************/
public void extractTermFrequencies()
{
   StringBuffer outCand = new StringBuffer();
   String freqFile = CORPUS_OUTPUT+"frequencies.txt";
   System.out.println("Extracting Frequencies");
   
  outCand.append("ID\t"+"TERM\t"+"TAG\t"+"Frequency\t"+System.getProperty("line.separator"));          
    //*************Count NN and put in Hash*************************
   for(int i=0;i< mymultiNNCandidate.size();i++)
    {
        int counter =0;
        for(int j=0;j< mymultiNNCandidate.size();j++)
        {
            if(mymultiNNCandidate.get(i).equals(mymultiNNCandidate.get(j)))
            counter++;
        }        
        Term value =new Term(mymultiNNCandidate.get(i));
        value.setTag(mymultiNNTag.get(i));
        value.setFrequency(counter);
        value.setCMCC("");
        Terms.put(mymultiNNCandidate.get(i),value);
    }
    System.out.println("After Single and Multi Nouns Addition, Terms Count : "+Terms.size());
    //*************Count JJNN and put in Hash*************************

    for(int i=0;i< myJJNNCandidate.size();i++)
    {
        int counter =0;
        for(int j=0;j< myJJNNCandidate.size();j++)
        {
            if(myJJNNCandidate.get(i).equals(myJJNNCandidate.get(j)))
            counter++;
        }        
        Term value =new Term(myJJNNCandidate.get(i));
        value.setTag(myJJNNTag.get(i));
        value.setFrequency(counter);
        value.setCMCC("");

        Terms.put(myJJNNCandidate.get(i),value);
    }
        System.out.println("After Adjective Nouns Addition, Terms Count : "+Terms.size());

      //*************Count NNCCNN and put in Hash*************************

    for(int i=0;i< myNNCCNNCandidate.size();i++)
    {
        int counter =0;
        for(int j=0;j< myNNCCNNCandidate.size();j++)
        {
            if(myNNCCNNCandidate.get(i).trim().equals(myNNCCNNCandidate.get(j).trim()))
            counter++;
            
        }    
        String[] mySplittedTokens = myNNCCNNCandidate.get(i).split(" ");

        Term value =new Term(myNNCCNNCandidate.get(i));
        value.setTag(myNNCCNNTag.get(i));
        value.setFrequency(counter);
        value.setCMCC(mySplittedTokens[1]);
        Terms.put(myNNCCNNCandidate.get(i),value);
    }
        System.out.println("After Added Nouns Addition, Terms Count : "+Terms.size());
      //*************Count NNCMNN and put in Hash*************************

    for(int i=0;i< myNNCMNNCandidate.size();i++)
    {
        int counter =0;
        for(int j=0;j< myNNCMNNCandidate.size();j++)
        {
            if(myNNCMNNCandidate.get(i).equals(myNNCMNNCandidate.get(j)))
            counter++;
            
        }        
        String[] mySplittedTokens = myNNCMNNCandidate.get(i).split(" ");

        Term value =new Term(myNNCMNNCandidate.get(i));
        value.setTag(myNNCMNNTag.get(i));
        value.setFrequency(counter);   
        value.setCMCC(mySplittedTokens[1]);
        Terms.put(myNNCMNNCandidate.get(i),value);
    }
        System.out.println("After Preposition Nouns Addition, Terms Count : "+Terms.size());

    
    /*****************************************************************/

    int ilocID=1;
    
    for(Iterator<String> it = Terms.keySet().iterator(); it.hasNext();)
    {
            String key = it.next();
            Term value = Terms.get(key);
            
            value.id =ilocID;
            
          // System.out.println(value.id +" "+ value.Tag+" "+value.Term+" "+ value.frequency);
            outCand.append(value.id+"\t"+value.Term+"\t"+value.Tag+"\t"+value.frequency+System.getProperty("line.separator"));          
            ilocID++;
      } 
    
         fileWriter(outCand,  freqFile);
}
/*****************************************************************/
public void extractTermNests()
{
    StringBuffer outCand = new StringBuffer();
    String nestFile = CORPUS_OUTPUT+"Nest.txt";
    System.out.println("Extracting Nest and freqNest");

    outCand.append("ID\t"+"TERM\t"+"TAG\t"+"Magnitude\t"+"Frequency\t"+"Nest\t"+"FreqNest\t"+"NestMap\t"+System.getProperty("line.separator"));          

     for(Iterator<String> it = Terms.keySet().iterator(); it.hasNext();)
    {

            String key      =    it.next();
            Term value      =    Terms.get(key);
            value.nest      =   0;
            value.freqNest  =   0;
       
           // System.out.println(value.id+" Count Nests of  : "+value.Term);          
            int counter =0;
            for(Iterator<String> ilt = Terms.keySet().iterator(); ilt.hasNext();)
            {
                 String lkey = ilt.next();
                 Term lvalue = Terms.get(lkey);
                 String[] mySplittedTokens = lvalue.Term.split(" ");
               //  System.out.println("Splitted Token Count"+lvalue.Term.split(" ").length);
                if(lvalue.Term.split(" ").length>1)
                {
                    for(String pattern : mySplittedTokens)
                    {
                        if(value.Term.equals(pattern))
                        {
                            counter++;
                            value.nestMap.put(lvalue.Term,lvalue.frequency);
                            value.freqNest=value.freqNest+ lvalue.frequency;
                        }
                    }
                }
            }
            value.nest   = counter;
      
      // System.out.println(value.id+"\t"+value.Term+"\t"+value.Tag+"\t"+value.Term.split(" ").length+"\t"+value.frequency+"\t"+value.nest+"\t"+value.freqNest);
            outCand.append(value.id+"\t"+value.Term+"\t"+value.Tag+"\t"+value.Term.split(" ").length+"\t"+value.frequency+"\t"+value.nest+"\t"+value.freqNest+"\t"+value.nestMap.toString()+System.getProperty("line.separator"));          
      } 
   
    fileWriter(outCand,  nestFile);
    
}
/****************************************************************/
public void calculateCVALUE()
{
   StringBuffer outCand = new StringBuffer();
   String cValueFile = CORPUS_OUTPUT+"CValue.txt";
  System.out.println("Calculating Cvalues of Terms");
  
  // outCand.append("ID\t"+"TERM\t"+"TAG\t"+"Magnitude\t"+"Frequency\t"+"Nest\t"+"FreqNest\t"+"NestMap\t"+"CVALUE"+System.getProperty("line.separator"));          
 outCand.append("ID\t"+"TERM\t"+"TAG\t"+"CVALUE"+System.getProperty("line.separator"));          

    for(Iterator<String> it = Terms.keySet().iterator(); it.hasNext();)
    {

            String key      =    it.next();
            Term result      =    Terms.get(key);
   
            double score;
            double log2a = Math.log((double) result.Term.split(" ").length + 0.1) / Math.log(2.0);
            double freqa = (double) result.frequency;
            
            double pTa = (double) result.nest;
            double sumFreqb = result.freqNest;

         
            score = pTa == 0 ? log2a * freqa : log2a * (freqa - (sumFreqb / pTa));

            result.setCValue(score);
           /// System.out.println(result.id+"\t"+result.Term+"\t"+result.Tag+"\t"+result.CValue+System.getProperty("line.separator"));          

            outCand.append(result.id+"\t"+result.Term+"\t"+result.Tag+"\t"+result.CValue+System.getProperty("line.separator"));          

        }
     fileWriter(outCand,  cValueFile);
}
/****************************************************************/
public void storeResultsDB()
{
    System.setProperty("file.encoding" , "UTF-8");
   StringBuffer outCand = new StringBuffer();
   String cValueFile = CORPUS_OUTPUT+"CValue.txt";
   System.out.println("Storing Results in DB");
   SQL_Handler storeDB= new SQL_Handler();
   storeDB.initialize_parameters("jdbc:mysql://localhost:3306/urduterms?useUnicode=true&characterEncoding=UTF-8","root","" );
   storeDB.connect_db();

   String[] fields={"termID","termUrdu","termTag","termFrequency","termNest","termCMCC","termNestMap","termCValue"};   

    for(Iterator<String> it = Terms.keySet().iterator(); it.hasNext();)
    {
        String key      =    it.next();
        Term result      =    Terms.get(key);
        //System.out.println(result.id+"\t"+result.Term+"\t"+result.Tag+"\t"+result.Term.split(" ").length+"\t"+result.frequency+"\t"+result.nest+"\t"+result.freqNest+"\t"+result.CValue);
        outCand.append(result.id+"\t"+result.Term+"\t"+result.Tag+"\t"+result.Term.split(" ").length+"\t"+result.frequency+"\t"+result.nest+"\t"+result.freqNest+"\t"+result.CValue+System.getProperty("line.separator"));
        String[] values={""+result.id,result.Term,result.Tag,String.valueOf(result.frequency),String.valueOf(result.nest),result.CMCC,result.nestMap.toString(),String.valueOf(result.CValue)};   
        
        storeDB.insert_table("terms",  fields,  values); 
    }
     fileWriter(outCand,  cValueFile);
}
/**********************************************************************/
public void fileWriter(String str, String path)
{
        try
        {
            File writeFile = new File(path);
            OutputStream fout = new FileOutputStream(writeFile);
            OutputStreamWriter writer = new OutputStreamWriter(fout, "UTF-8");
            writer.write(str);
            writer.close();
            fout.close();
        }
        catch (Exception ex)
        {
            System.out.println("File Writer Value of Exception is :" + ex.getMessage());
        }
    }
/**********************************************************************/
public void fileWriter(StringBuffer stringBuffer, String output_path)
{
        fileWriter(stringBuffer.toString(),output_path);
}
/*********************************************************************/    
public StringBuffer fileReader(File inputFile)
{
        StringBuffer contents = new StringBuffer();

        BufferedReader input = null;
        try
        {
            input = new BufferedReader(new FileReader(inputFile));
            String line = null;
            while((line = input.readLine()) != null)
            {
                contents.append(line);
            }
        }
        catch (FileNotFoundException ex)
        {
            ex.printStackTrace();
        }
        catch (IOException ex)
        {
            ex.printStackTrace();
        }
        finally
        {
            try
            {
                if(input != null)
                {
                    input.close();
                }
            }
            catch (IOException ex)
            {
                ex.printStackTrace();
            }
        }
                

        return contents;
    }
/*********************************************************************/
public static void main(String[] args)
{
        
        try
        {
                urduTermRecognizer CURLURDUtoTWOColumn = new urduTermRecognizer();
                CURLURDUtoTWOColumn.convert_to_Two_Column_text();
                
                CURLURDUtoTWOColumn.extractmultiNNCandidate();
                CURLURDUtoTWOColumn.extractJJNNCandidate();   
                CURLURDUtoTWOColumn.extractNNCCNNCandidate();
                CURLURDUtoTWOColumn.extractNNCMNNCandidate();
                
                CURLURDUtoTWOColumn.extractTermFrequencies();
                CURLURDUtoTWOColumn.extractTermNests();
                CURLURDUtoTWOColumn.calculateCVALUE();   
                CURLURDUtoTWOColumn.storeResultsDB();   
                         
        }
        catch (Exception ex)
        {
            System.out.println("Main Value of Exception in reading input is : " + ex.getMessage());
        }
    }
/*********************************************************************/    
}
